# -*- coding: utf-8 -*-
import scrapy

import re
import json
import logging
from copy import copy, deepcopy
from FDC_spider.utils.transfer_time import transfer_date
from FDC_spider.items import FdcEstateGuidItem, FdcBuildingItem, FdcRoomItem
from FDC_spider.constants import ZHSH_DISTRICT_PARAMS_DICT, ZHSH_USE_FACT_DICT, ZHSH_BUILD_STRUCT_DICT

logger = logging.getLogger(__name__)


class ZhoushanSpider(scrapy.Spider):
    name = 'zhoushan'
    allowed_domains = ['zstmsf.com']
    start_urls = ['http://www.zstmsf.com/website/app/ProjectIndex/projectSearch']
    project_li_url = 'http://www.zstmsf.com/website/app/ProjectIndex/searchProjectDetail/'  # POST 项目列表url
    project_detail_url = 'http://www.zstmsf.com/website/app/SpfProjectIndex/projectMainDetailInfo/'  # POST 项目详情url
    permit_li_url = 'http://www.zstmsf.com/website/app/SpfProjectIndex/donePresell/'  # POST 预售许可证列表url
    building_li_url = 'http://www.zstmsf.com/website/app/SpfProjectIndex/presellBuilding/'  # POST 楼栋列表url
    building_detail_url = 'http://www.zstmsf.com/website/app/SpfProjectIndex/buildingTz/'  # POST 楼栋详情url
    room_li_url_temp = 'http://www.zstmsf.com/website/app/buildingtable/tableData?id={}&building_ID={}&sort=asc'  # GET 房号列表url

    def parse(self, response):
        """
        获取行政区/县
        :param response:
        :return:
        """
        district_str_li = response.xpath(
            "//div[@class='lp_ordertab']/div[contains(@class,'lp_code')]/a//text()").extract()
        # 获取行政区/县列表
        district_li = self._get_district_li(district_str_li)
        for district_name in district_li:
            try:
                # 获取不同行政区/县对应请求参数
                division_code = ZHSH_DISTRICT_PARAMS_DICT.get(district_name, None)
                assert division_code
            except:
                logger.error('{} 获取请求参数出错'.format(district_name))
            else:
                # 构造POST请求，获取项目列表
                data = {
                    'divisioncode': division_code,
                    'open_time': 'OPEN_TIME',
                    'keyword': '',
                    'OrderBy': '',
                    'pagesize': '8',
                    'pageindex': '1',
                }
                yield scrapy.FormRequest(
                    self.project_li_url,
                    formdata=data,
                    callback=self.parse_project_li,
                    meta=dict(page_num=1, district_name=district_name, division_code=division_code, ),
                    priority=1,
                )

    def parse_project_li(self, response):
        """
        获取项目列表
        :param response:
        :return:
        """
        page_num = copy(response.meta['page_num'])
        district_name = copy(response.meta['district_name'])
        division_code = copy(response.meta['division_code'])
        try:
            # 反序列化响应内容
            resp_dict = json.loads(response.body.decode())
            # 响应状态
            success = resp_dict.get('success')
            assert success, '{}-第{}页  响应状态码出错'.format(district_name, page_num)
            # 项目总数
            project_total_num = int(resp_dict.get('total'))
            assert project_total_num, '{}-第{}页 项目总数获取出错'.format(district_name, page_num)
            # 当前页项目列表
            project_li = resp_dict.get('rows')
            assert project_li, '{}-第{}页 项目列表获取为空'.format(district_name, page_num)
        except AssertionError as e:
            logger.error(e)
        except:
            logger.error('{}-第{}页 项目列表获取出错'.format(district_name, page_num))
        else:
            for i, project_dict in enumerate(project_li):
                item_eg = FdcEstateGuidItem()
                item_eg['projectId'] = project_dict.get('PROJECTALL_ID', None)
                item_eg['districtName'] = district_name
                presell_id = project_dict.get('PRESELL_ID', None)
                if item_eg['projectId'] and presell_id:
                    # 获取项目详情
                    referer_url_temp = 'http://www.zstmsf.com/website/app/SpfProjectIndex/projectIndexMain?id={}&presellid={}'
                    headers = {
                        'Referer': referer_url_temp.format(item_eg['projectId'], presell_id)
                    }
                    data = dict(projectId=item_eg['projectId'])
                    yield scrapy.FormRequest(
                        self.project_detail_url,
                        headers=headers,
                        formdata=data,
                        callback=self.parse_project_detail,
                        meta=dict(item_eg=deepcopy(item_eg), referer_url=deepcopy(headers['Referer']), ),
                        priority=2,
                    )
                else:
                    logger.error(f'第{page_num}页-第{i + 1}个 项目id获取失败')

            # 翻页
            page_total_num = project_total_num // 8 + 1 if project_total_num % 8 else project_total_num // 8
            for page_num in range(2, page_total_num + 1):
                data = {
                    'divisioncode': division_code,
                    'open_time': 'OPEN_TIME',
                    'keyword': '',
                    'OrderBy': '',
                    'pagesize': '8',
                    'pageindex': f'{page_num}',
                }
                yield scrapy.FormRequest(
                    self.project_li_url,
                    formdata=data,
                    callback=self.parse_project_li,
                    meta=dict(page_num=page_num, district_name=district_name, division_code=division_code, ),
                    priority=1,
                )

    def parse_project_detail(self, response):
        """
        获取项目详情
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        referer_url = copy(response.meta['referer_url'])
        try:
            # 反序列化响应内容
            resp_dict = json.loads(response.body.decode())
            # 响应状态
            success = resp_dict.get('success', False)
            # 提取项目list
            project_li = resp_dict.get('rows', list())
            # 提取项目详情dict
            project_dict = project_li[0]
            assert success and len(project_li) == 1 and project_dict, '{}-{}  项目详情dict提取出错'.format(
                item_eg['districtName'], item_eg['projectId'])
        except AssertionError as e:
            logger.error(e)
        except:
            logger.error('{}-{} 项目详情信息获取出错'.format(item_eg['districtName'], item_eg['projectId']))
        else:
            item_eg['projectName'] = project_dict.get('PROJECTALL_NAME', None)
            item_eg['projectAddress'] = project_dict.get('PROJECTALL_ADDRESS', None)
            item_eg['developerName'] = project_dict.get('ENTERPRISE_NAME', None)
            item_eg['lngBd'] = project_dict.get('COORDINATES_Y', None)
            item_eg['latBd'] = project_dict.get('COORDINATES_X', None)
            item_eg['salesTel'] = project_dict.get('SELL_TELEPHONE', None)
            item_eg['projectBrief'] = project_dict.get('PROJECTALL_INFO', None)

            # 获取预售许可证列表
            headers = {
                'Referer': referer_url,
            }
            data = dict(projectId=item_eg['projectId'])
            yield scrapy.FormRequest(
                self.permit_li_url,
                headers=headers,
                formdata=data,
                callback=self.parse_peimit_li,
                meta=dict(item_eg=deepcopy(item_eg), ),
                priority=3,
            )

    def parse_peimit_li(self, response):
        """
        获取预售许可证列表
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        try:
            # 反序列化响应内容
            resp_dict = json.loads(response.body.decode())
            # 响应状态
            success = resp_dict.get('success', False)
            assert success, '{}-{}-{} 预售许可证列表状态码错误'.format(item_eg['districtName'], item_eg['projectName'],
                                                           item_eg['projectId'])
            # 提取预售许可证list
            permit_li = resp_dict.get('rows', list())
            assert permit_li, '{}-{}-{} 预售许可证列表提取为空'.format(item_eg['districtName'], item_eg['projectName'],
                                                            item_eg['projectId'])
        except AssertionError as e:
            logger.error(e)
        except:
            logger.error(
                '{}-{}-{} 预售许可证列表获取出错'.format(item_eg['districtName'], item_eg['projectName'], item_eg['projectId']))
        else:
            # 遍历获取预售许可证详情
            for permit_dict in permit_li:
                item_eg['projectSubName'] = permit_dict.get('PROJECT_NAME', None)
                item_eg['completionDate'] = permit_dict.get('FINISH_DATE', None)
                item_eg['preSalePermit'] = permit_dict.get('PRESELL_NAME', None)
                item_eg['permitId'] = permit_dict.get('PRESELL_ID', None)
                item_eg['permitTotalRoomNum'] = permit_dict.get('PRESELL_COUNT', None)
                item_eg['permitTotalArea'] = permit_dict.get('PRESELL_AREA', None)
                item_eg['permitSaleableRoomNum'] = permit_dict.get('KSTS', None)
                item_eg['propertyType'] = permit_dict.get('PROJECT_USE', None)
                opening_date_stamp = permit_dict.get('OPEN_TIME', None)
                item_eg['openingDate'] = self._get_date(opening_date_stamp)
                cert_date_stamp = permit_dict.get('PRESELL_CREATEDATE', None)
                item_eg['certDate'] = self._get_date(cert_date_stamp)
                yield item_eg

                if item_eg['permitId']:
                    # 获取楼栋列表
                    referer_url_temp = 'http://www.zstmsf.com/website/app/SpfProjectIndex/projectInfoPresell?presellid={}&id={}'
                    headers = {
                        'Referer': referer_url_temp.format(item_eg['permitId'], item_eg['projectId'])
                    }
                    data = dict(presellId=item_eg['permitId'])
                    yield scrapy.FormRequest(
                        self.building_li_url,
                        headers=headers,
                        formdata=data,
                        callback=self.parse_building_li,
                        meta=dict(item_eg=deepcopy(item_eg), ),
                        priority=4,
                    )
                else:
                    logger.warning(
                        '{}-{}-{} 预售许可证id提取为空，无法获取楼栋信息'.format(item_eg['districtName'], item_eg['projectName'],
                                                               item_eg['projectId']))

    def parse_building_li(self, response):
        """
        获取楼栋列表
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        try:
            # 反序列化响应内容
            resp_dict = json.loads(response.body.decode())
            # 响应状态
            success = resp_dict.get('success', False)
            assert success, '{}-{}-{}-{} 楼栋列表状态码错误'.format(item_eg['districtName'], item_eg['projectName'],
                                                           item_eg['projectId'], item_eg['preSalePermit'])
            # 提取楼栋列表
            building_li = resp_dict.get('rows', list())
            assert building_li, '{}-{}-{}-{} 楼栋列表提取为空'.format(item_eg['districtName'], item_eg['projectName'],
                                                              item_eg['projectId'], item_eg['preSalePermit'])
        except AssertionError as e:
            logger.error(e)
        except:
            logger.error(
                '{}-{}-{}-{} 楼栋列表获取出错'.format(item_eg['districtName'], item_eg['projectName'], item_eg['projectId'],
                                              item_eg['preSalePermit']))
        else:
            # 遍历获取楼栋dict
            for building_dict in building_li:
                item_bd = FdcBuildingItem()
                item_bd['districtName'] = item_eg['districtName']
                item_bd['projectName'] = item_eg['projectName']
                item_bd['projectId'] = item_eg['projectId']
                item_bd['preSalePermit'] = item_eg['preSalePermit']
                item_bd['permitId'] = item_eg['permitId']
                item_bd['buildingId'] = building_dict.get('BUILDING_ID', None)
                item_bd['buildingNo'] = building_dict.get('BUILDING_NUMBER', None)
                item_bd['saleableRoomNo'] = building_dict.get('KSCOUNT', None)
                item_bd['soldRoomNo'] = building_dict.get('YSCOUNT', None)
                item_bd['reserveRoomNo'] = building_dict.get('YDCOUNT', None)

                # 获取楼栋详情
                referer_url_temp = 'http://www.zstmsf.com/website/app/SpfProjectIndex/projectInfoLoupanbiao?buildingId={}&id={}'
                if item_bd['buildingId']:
                    headers = {
                        'Referer': referer_url_temp.format(item_bd['buildingId'], item_bd['projectId']),
                    }
                    data = dict(buildingId=item_bd['buildingId'])
                    yield scrapy.FormRequest(
                        self.building_detail_url,
                        headers=headers,
                        formdata=data,
                        callback=self.parse_building_detail,
                        meta=dict(item_bd=deepcopy(item_bd), referer_url=deepcopy(headers['Referer']), ),
                        priority=5,
                    )
                else:
                    logger.error('{}-{}-{}-{} 楼栋id获取出错，楼栋详情无法获取'.format(item_bd['districtName'], item_bd['projectName'],
                                                                        item_bd['preSalePermit'],
                                                                        item_bd['buildingNo']))

    def parse_building_detail(self, response):
        """
        获取楼栋详情
        :param response:
        :return:
        """
        item_bd = copy(response.meta['item_bd'])
        referer_url = copy(response.meta['referer_url'])
        try:
            # 反序列化响应内容
            resp_dict = json.loads(response.body.decode())
            # 响应状态
            success = resp_dict.get('success', False)
            assert success, '{}-{}-{}-{} 楼栋详情响应状态码出错'.format(item_bd['districtName'], item_bd['projectName'],
                                                             item_bd['preSalePermit'], item_bd['buildingNo'])
            # 提取楼栋详情list
            building_li = resp_dict.get('rows', list())
            # 楼栋详情dict
            building_dict = building_li[0]
            assert len(building_li) == 1 and building_dict, '{}-{}-{}-{} 楼栋详情响应提取出错'.format(item_bd['districtName'],
                                                                                            item_bd['projectName'],
                                                                                            item_bd['preSalePermit'],
                                                                                            item_bd['buildingNo'])
        except AssertionError as e:
            logger.error(e)
        except:
            logger.error(
                '{}-{}-{}-{} 楼栋详情获取失败'.format(item_bd['districtName'], item_bd['projectName'], item_bd['preSalePermit'],
                                              item_bd['buildingNo']))
        else:
            item_bd['buildingLocation'] = building_dict.get('BUILDING_ADDRESS', None)
            item_bd['buildingTotalArea'] = building_dict.get('ALLAREA', None)
            item_bd['buildingSaleableArea'] = building_dict.get('KSAREA', None)
            item_bd['avgPrice'] = building_dict.get('ZJJ', None)

            # 获取房号列表
            headers = {
                'Referer': referer_url,
            }
            yield scrapy.Request(
                self.room_li_url_temp.format(item_bd['projectId'], item_bd['buildingId'], ),
                headers=headers,
                callback=self.parse_room_li,
                meta=dict(item_bd=deepcopy(item_bd), ),
                priority=6,
            )

    def parse_room_li(self, response):
        """
        获取房号列表
        :param response:
        :return:
        """
        item_bd = copy(response.meta['item_bd'])
        try:
            # 反序列化响应内容
            resp_dict = json.loads(response.body.decode())
            # 提取楼栋单元列表
            bd_unit_li = resp_dict.get('buildinfo', None)
            # 提取楼栋总层数
            bd_floor_dict = resp_dict.get('cs', None)
            # 提取房号列表
            room_li = resp_dict.get('houses', None)
            assert room_li
        except:
            logger.error(
                '{}-{}-{}-{} 房号列表提取出错'.format(item_bd['districtName'], item_bd['projectName'], item_bd['preSalePermit'],
                                              item_bd['buildingNo']))
        else:
            item_bd['unitLi'] = self._get_unit_li(bd_unit_li)
            item_bd['floorTotalNo'] = self._get_total_floor_num(bd_floor_dict)
            yield item_bd

            # 遍历获取房号dict
            for room_dict in room_li:
                item_rm = FdcRoomItem()
                item_rm['projectName'] = item_bd['projectName']
                item_rm['projectId'] = item_bd['projectId']
                item_rm['buildingNo'] = item_bd['buildingNo']
                item_rm['roomNo'] = room_dict.get('ROOM_NUMBER', None)
                item_rm['roomId'] = room_dict.get('HOUSE_ID', None)
                item_rm['roomLocation'] = room_dict.get('HOUSE_ADDRESS', None)
                item_rm['unitNo'] = room_dict.get('UNIT_NUMBER', None)
                # item_rm['unitNo'] = item_rm['unitNo'] if item_rm['unitNo'] and item_rm['unitNo'] != '0' else '1'
                item_rm['referencePrice'] = room_dict.get('ONEPRICE', None)
                item_rm['roomFloor'] = room_dict.get('FLOOR_REALRIGHT', None)
                item_rm['forecastBuildArea'] = room_dict.get('PREBUILD_AREA', None)
                item_rm['forecastInnerArea'] = room_dict.get('PREINSIDE_AREA', None)
                item_rm['forecastSharedArea'] = room_dict.get('PRESHARE_AREA', None)
                item_rm['roomUse'] = self._get_room_use(room_dict.get('USE_FACT', None))
                item_rm['buildingStructure'] = self._get_room_building_structure(room_dict.get('BUILD_STRUCT', None))
                # 构造解析字典提取销售状态
                sale_status_dict = dict()
                sale_status_dict['SALE_STATE'] = room_dict.get('SALE_STATE', None)
                sale_status_dict['YXBZ'] = room_dict.get('YXBZ', None)
                sale_status_dict['YDBZ'] = room_dict.get('YDBZ', None)
                sale_status_dict['DYZT'] = room_dict.get('DYZT', None)
                sale_status_dict['XZZT'] = room_dict.get('XZZT', None)
                sale_status_dict['CFZT'] = room_dict.get('CFZT', None)
                item_rm['saleStatus'] = self._get_sale_status(sale_status_dict)
                yield item_rm

    @staticmethod
    def _get_district_li(_li: list) -> list:
        """
        提取行政区/县列表
        :param _li:
        :return:
        """
        regex = re.compile(r'[\u4e00-\u9fa5]+')
        district_li = list()
        try:
            for district_str in _li:
                district_ret = regex.findall(district_str)
                if district_ret:
                    district_li.append(district_ret[0])
            assert district_li
        except:
            logger.error('行政区/县列表提取出错')
            return ['新城', '定海', '普陀', '岱山', '嵊泗']  # 返回一个默认值，防止提取出错无法正常爬取
        else:
            return district_li

    @staticmethod
    def _get_date(_str: str) -> str or None:
        """
        将时间戳字符串转换为date字符串
        :param _str:
        :return:
        """
        try:
            ret = transfer_date(_str)
        except Exception as e:
            logger.error(f'时间戳字符串转换出错，error_msg:{e}')
        else:
            return ret

    @staticmethod
    def _get_unit_li(_li: list) -> list:
        """
        提取单元列表
        :param _li:
        :return:
        """
        unit_li = list()
        try:
            for unit_dict in _li:
                unit_num = unit_dict.get('UNIT_NUMBER', None)
                unit_li.append(unit_num)
        except Exception as e:
            logger.error(f'楼栋单元列表提取出错，error_msg:{e}')
            return ['1']
        else:
            return unit_li

    @staticmethod
    def _get_total_floor_num(_dict: dict) -> str or None:
        """
        提取总层数
        :param _dict:
        :return:
        """
        try:
            ret = _dict.get('CS')
            assert ret, '总层数提取为空'
        except AssertionError as e:
            logger.warning(e)
        except Exception as e:
            logger.error(f'楼栋总楼层提取出错，error_msg:{e}')
        else:
            return ret

    @staticmethod
    def _get_room_use(_str: str) -> str or None:
        """
        提取房屋用途
        :param _str:
        :return:
        """
        try:
            ret = ZHSH_USE_FACT_DICT.get(str(_str), None)
            assert ret, '房屋用途提取为空'
        except AssertionError as e:
            logger.warning(e)
        except Exception as e:
            logger.error(f'房屋用途提取出错，error_msg:{e}')
        else:
            return ret

    @staticmethod
    def _get_room_building_structure(_str: str) -> str or None:
        """
        提取建筑结构
        :param _str:
        :return:
        """
        try:
            ret = ZHSH_BUILD_STRUCT_DICT.get(str(_str), None)
            assert ret, '房屋建筑结构提取为空'
        except AssertionError as e:
            logger.warning(e)
        except Exception as e:
            logger.error(f'房屋建筑结构提取出错，error_msg:{e}')
        else:
            return ret

    @staticmethod
    def _get_sale_status(_dict: dict) -> str or None:
        """
        提取房屋销售状态
        :param _dict:
        :return:
        """
        try:
            if _dict['SALE_STATE'] == 1015 and _dict['YXBZ'] == 1:
                return '可售'
            if _dict['YDBZ'] != 0:
                return '预定'
            if _dict['DYZT'] == 1:
                return '抵押'
            if _dict['SALE_STATE'] == 1025 or _dict['SALE_STATE'] == 1030:
                return '已售'
            if _dict['XZZT'] == 1 and _dict['SALE_STATE'] < 1025:
                return '已限制'
            if _dict['CFZT'] == 1 and _dict['SALE_STATE'] < 1025:
                return '查封'
        except Exception as e:
            logger.error(f'房号销售状态提取出错，error_msg"{e}')
